package com.example.demo.componet.spark;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

import java.util.ArrayList;
import java.util.List;

class sparkConnect {

    sparkConnect() {

        //解决java.io.FileNotFoundException: HADOOP_HOME and hadoop.home.dir are unset

        System.setProperty("hadoop.home.dir","D:\\SoftWares\\Apache\\spark-3.3.1-bin-hadoop3");


        JavaSparkContext sc = new JavaSparkContext("local", "thisSpark");
        List<Integer> list = new ArrayList<Integer>();
        list.add(1); list.add(2); list.add(3);

        //内存中获取数据常见rdd
        JavaRDD<Integer> rdd = sc.parallelize(list);

        //rdd计算
        JavaRDD<Integer> listPlus = rdd.map(y->y+5);

        System.out.println("list is:" + list);
        System.out.println("listPlus is:"+ listPlus);
        System.out.println("the relation is list plus five = listPlus");


    }

    public static void main(String[] args) {

        sparkConnect sparkConnect = new sparkConnect();
    }

}


